{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.chdir('../')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "from data_utils.basic_data_round2 import load_basic_dataset\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "basic dataset len: 12999\n"
     ]
    }
   ],
   "source": [
    "train_df = load_basic_dataset('train')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "title_texts_entity = train_df[['title','text','entity']].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 320,
   "metadata": {},
   "outputs": [],
   "source": [
    "def text_startwith_title(title,text,threshold):\n",
    "    if isinstance(title,float):\n",
    "        return True\n",
    "    title_set = set(title)\n",
    "    text_set = set(text[:len(title)])\n",
    "    common = len(title_set& text_set)/len(title_set)\n",
    "    return common > threshold\n",
    "\n",
    "def get_text_preprocessed(title, text, entities, max_text_length, entity_start_char, entity_end_char):\n",
    "    if not text_startwith_title(title, text, 0.9):\n",
    "        text = title + '\\n' + text\n",
    "    text = text[:max_text_length]\n",
    "    pattern = re.compile(r\"[^\\u4e00-\\u9fa5@?（）【】《》“”‘’#？()[];:；：。.、]\")\n",
    "    text = re.sub(pattern, ' ', text)\n",
    "    index_pairs = []\n",
    "    for entity in entities.split(';'):\n",
    "        matched_pair = [sub.span() for sub in\n",
    "                        re.finditer(entity.replace(r'?', r'\\?').replace(r'*',r'\\*').replace(r'(', r'\\(').replace(r')', r'\\)'), text)]\n",
    "        index_pairs += matched_pair\n",
    "    index_pairs.sort()\n",
    "    maxrange_index_pairs = []\n",
    "    if len(index_pairs) == 0:\n",
    "        return text, [None]\n",
    "    start, end = index_pairs[0]\n",
    "    for pair in index_pairs:\n",
    "        start1, end1 = pair\n",
    "        if end1 <= end:\n",
    "            start = min(start, start1)\n",
    "        else:\n",
    "            maxrange_index_pairs.append((start, end))\n",
    "            start = start1\n",
    "            end = end1\n",
    "    if end1 <= end:\n",
    "        start1 = min(start, start1)\n",
    "    maxrange_index_pairs.append((start1, end))\n",
    "    conflict_pairs = []\n",
    "    for i in range(len(maxrange_index_pairs) - 1):\n",
    "        start1, end1 = maxrange_index_pairs[i]\n",
    "        start2, end2 = maxrange_index_pairs[i + 1]\n",
    "        if start2 < end1:\n",
    "            conflict_pairs.append((start2, end2))\n",
    "\n",
    "    off = 0\n",
    "    text = list(text)\n",
    "    pair_entities = []\n",
    "    for pair in maxrange_index_pairs:\n",
    "        if pair not in conflict_pairs:\n",
    "            text.insert(pair[0] + off, entity_start_char)\n",
    "            off += 1\n",
    "            entity = ''.join(text[pair[0] + off:pair[1] + off])\n",
    "            pair_entities.append(entity)\n",
    "            text.insert(pair[1] + off, entity_end_char)\n",
    "            off += 1\n",
    "        print(conflict_pairs)\n",
    "    return ''.join(text), pair_entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "metadata": {},
   "outputs": [],
   "source": [
    "title,text,entities = title_texts_entity[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('????发表了博文《【[CLES]富金利[SEEP]】9月[CLES]富金利[SEEP]教你如何安全提高最大化收益！》网络[CLES]理财时代[SEEP]参与互联网投资理财选择正规的理财平台是理财关键，很多人都因为没有经受住高收益的诱惑不进行理性投资而遭遇了欺骗，微弱，而对网上投资理【[CLES]富金利[SEEP]】9月[CLES]富金利[SEEP]教你如何安全提高最大化收益！ ?????',\n",
       " ['富金利', '富金利', '理财时代', '富金利', '富金利'])"
      ]
     },
     "execution_count": 322,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_text_preprocessed(title,text,entities,max_text_length=400,entity_start_char='[CLES]',entity_end_char='[SEEP]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 323,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'????【利空公告】$上证指数(SH000001)$?   *ST雏鹰：公司股票已连续13个交易日收盘价格均低于股票面值（即1元），公司股票可能将被终止上市  航天通信：子公司部分银行债务逾期，逾期债务本金约5.6亿元  老百姓：股东泽星投资拟清仓减持不超29.78%股份$老百姓(SH603883)$?   先锋新材：股东贺兰英拟减持 ?????'"
      ]
     },
     "execution_count": 323,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entity = '????宜湃网'\n",
    "title,text,entities  = train_df[train_df['id']==11207][['title','text','entity']].values[0]\n",
    "title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'星投资;*ST雏鹰;泽星投资;老百姓;航天通信;先锋新材'"
      ]
     },
     "execution_count": 318,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'????宜湃网'"
      ]
     },
     "execution_count": 279,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r'????宜湃网'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'????宜湃网'"
      ]
     },
     "execution_count": 284,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
